import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import numpy as np
import cv2
import glob
%matplotlib inline
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import glob
from moviepy.editor import VideoFileClip
import os
from mpl_toolkits.mplot3d import Axes3D
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC
%matplotlib inline
# Read in cars and notcars
cars = glob.glob('vehicles/*/*.png')
notcars = glob.glob('non-vehicles/*/*.png')
# Just for fun choose random car / not-car indices and plot example images
car_ind = np.random.randint(0, len(cars))
notcar_ind = np.random.randint(0, len(notcars))
# Read in car / not-car images
car_image = mpimg.imread(cars[car_ind])
notcar_image = mpimg.imread(notcars[notcar_ind])
print ('Number of car images: ', len(cars))
print ('Number of non-car images: ', len(cars))
# Plot the examples
fig = plt.figure()
plt.subplot(121)
plt.imshow(car_image)
plt.title('Example Car Image')
plt.subplot(122)
plt.imshow(notcar_image)
plt.title('Example Not-car Image')
# Save images
fig.savefig('output_images/data_exploration.png')
# Define a function to compute color histogram features
def color_hist(img, nbins=32, bins_range=(0, 256)):
# Compute the histogram of the color channels separately
rhist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
ghist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
bhist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
# Generating bin centers
bin_edges = rhist[1]
bin_centers = (bin_edges[1:] + bin_edges[0:len(bin_edges)-1])/2
# Concatenate the histograms into a single feature vector
hist_features = np.concatenate((rhist[0],ghist[0],bhist[0]))
# Return the individual histograms, bin_centers and feature vector
return rhist, ghist, bhist, bin_centers, hist_features
test_image = cv2.imread('test_images/car.png')
rh, gh, bh, bincen, feature_vec = color_hist(test_image, nbins=32, bins_range=(0, 256))
# Plot a figure with all three bar charts
if rh is not None:
fig = plt.figure(figsize=(12,3))
plt.subplot(131)
plt.bar(bincen, rh[0])
plt.xlim(0, 256)
plt.title('R Histogram')
plt.subplot(132)
plt.bar(bincen, gh[0])
plt.xlim(0, 256)
plt.title('G Histogram')
plt.subplot(133)
plt.bar(bincen, bh[0])
plt.xlim(0, 256)
plt.title('B Histogram')
fig.tight_layout()
fig.savefig('output_images/color_hist_car.png')
else:
print('Your function is returning None for at least one variable...')
non_car_image = cv2.imread('test_images/non_car.png')
rh, gh, bh, bincen, feature_vec = color_hist(non_car_image, nbins=32, bins_range=(0, 256))
# Plot a figure with all three bar charts
if rh is not None:
fig = plt.figure(figsize=(12,3))
plt.subplot(131)
plt.bar(bincen, rh[0])
plt.xlim(0, 256)
plt.title('R Histogram')
plt.subplot(132)
plt.bar(bincen, gh[0])
plt.xlim(0, 256)
plt.title('G Histogram')
plt.subplot(133)
plt.bar(bincen, bh[0])
plt.xlim(0, 256)
plt.title('B Histogram')
fig.tight_layout()
fig.savefig('output_images/color_hist_non_car.png')
else:
print('Your function is returning None for at least one variable...')
def convert_color(img, conv='RGB2YCrCb'):
if conv == 'RGB2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
if conv == 'BGR2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
if conv == 'RGB2LUV':
return cv2.cvtColor(img, cv2.COLOR_RGB2LUV)
# Define a function to draw bounding boxes
def draw_boxes(img,boxes,color=(0,0,255),thick=6):
draw_img=np.copy(img)
for bbox in boxes:
cv2.rectangle(draw_img,bbox[0],bbox[1],color,thick)
return draw_img
def plot3d(pixels, colors_rgb,
axis_labels=list("RGB"), axis_limits=[(0, 255), (0, 255), (0, 255)]):
"""Plot pixels in 3D."""
# Create figure and 3D axes
fig = plt.figure(figsize=(8, 8))
ax = Axes3D(fig)
# Set axis limits
ax.set_xlim(*axis_limits[0])
ax.set_ylim(*axis_limits[1])
ax.set_zlim(*axis_limits[2])
# Set axis labels and sizes
ax.tick_params(axis='both', which='major', labelsize=14, pad=8)
ax.set_xlabel(axis_labels[0], fontsize=16, labelpad=16)
ax.set_ylabel(axis_labels[1], fontsize=16, labelpad=16)
ax.set_zlabel(axis_labels[2], fontsize=16, labelpad=16)
# Plot pixel values with colors given in colors_rgb
ax.scatter(
pixels[:, :, 0].ravel(),
pixels[:, :, 1].ravel(),
pixels[:, :, 2].ravel(),
c=colors_rgb.reshape((-1, 3)), edgecolors='none')
return ax # return Axes3D object for further manipulation
# Read a color image
img = cv2.imread('test_images/car.png')
# Select a small fraction of pixels to plot by subsampling it
scale = max(img.shape[0], img.shape[1], 64) / 64 # at most 64 rows and columns
img_small = cv2.resize(img, (np.int(img.shape[1] / scale), np.int(img.shape[0] / scale)), interpolation=cv2.INTER_NEAREST)
# Convert subsampled image to desired color space(s)
img_small_RGB = cv2.cvtColor(img_small, cv2.COLOR_BGR2RGB) # OpenCV uses BGR, matplotlib likes RGB
img_small_HSV = cv2.cvtColor(img_small, cv2.COLOR_BGR2HSV)
img_small_YCrCb = cv2.cvtColor(img_small, cv2.COLOR_RGB2YCrCb)
img_small_rgb = img_small_RGB / 255. # scaled to [0, 1], only for plotting
# Plot and show
plot3d(img_small_RGB, img_small_rgb)
plt.show()
plot3d(img_small_HSV, img_small_rgb, axis_labels=list("HSV"))
plt.show()
plot3d(img_small_YCrCb, img_small_rgb, axis_labels=list("YCrCb"))
plt.show()
While it could be cumbersome to include three color channels of a full resolution image, we can perform spatial binning on an image and still retain enough information to help in finding vehicles.
# # Define a function to compute color histogram features
# def bin_spatial(img, size=(32, 32)):
# # Use cv2.resize().ravel() to create the feature vector
# features = cv2.resize(img, size).ravel()
# # Return the feature vector
# return features
# img = cv2.imread('test_images/car.png')
# feature_vec = bin_spatial(img, size=(32, 32))
# # Plot features
# plt.plot(feature_vec)
# plt.title('Spatially Binned Features')
# plt.savefig('output_images/bin_spatial.png')
def bin_spatial(img, color_space='HSV', size=(32, 32)):
# Convert image to new color space (if specified)
if color_space != 'RGB':
if color_space == 'HSV':
feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
elif color_space == 'LUV':
feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2LUV)
elif color_space == 'HLS':
feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2HLS)
elif color_space == 'YUV':
feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
elif color_space == 'YCrCb':
feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
else: feature_image = np.copy(img)
# Use cv2.resize().ravel() to create the feature vector
features = cv2.resize(feature_image, size).ravel()
# Return the feature vector
return features
img = cv2.imread('test_images/car.png')
feature_vec = bin_spatial(img, color_space='HSV', size=(32, 32))
# Plot features
plt.plot(feature_vec)
plt.title('Spatially Binned Features')
plt.show()
from skimage.feature import hog
# Define a function to return HOG features and visualization
def get_hog_features(img, orient, pix_per_cell, cell_per_block, vis=False, feature_vec=True):
if vis == True:
# Use skimage.hog() to get both features and a visualization
features, hog_image = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block), visualise=vis, feature_vector=feature_vec)
return features, hog_image
else:
# Use skimage.hog() to get features only
features = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block), visualise=vis, feature_vector=feature_vec)
return features
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Define HOG parameters
orient = 9
pix_per_cell = 8
cell_per_block = 2
# Call our function with vis=True to see an image output
features, hog_image = get_hog_features(gray, orient, pix_per_cell, cell_per_block, vis=True, feature_vec=False)
# Plot the examples
fig = plt.figure()
plt.subplot(121)
plt.imshow(gray, cmap='gray')
plt.title('Example Car Image')
plt.subplot(122)
plt.imshow(hog_image, cmap='gray')
plt.title('HOG Visualization')
# save images
fig.savefig('output_images/hog_features.png')
from sklearn.preprocessing import StandardScaler
# Define a function to extract features from a list of images
def extract_features(imgs, cspace='RGB', spatial_size=(32, 32),
hist_bins=32, hist_range=(0, 256),
orient=9, pix_per_cell=8, cell_per_block=2, hog_channel=0,
spatial_feat=True, hist_feat=True, hog_feat=True):
# Create a list to append feature vectors to
features = []
# Iterate through the list of images
for file in imgs:
combined_features = []
# Read in each one by one
image = mpimg.imread(file, format='PNG')
# apply color conversion if other than 'RGB'
feature_image = convert_color(image, conv='RGB2YCrCb')
if spatial_feat:
# Apply bin_spatial() to get spatial color features
spatial_features = bin_spatial(feature_image, size=spatial_size)
combined_features.append(spatial_features)
if hist_feat:
# Apply color_hist() also with a color space option now
rhist, ghist, bhist, bin_centers, hist_features = color_hist(feature_image, nbins=hist_bins, bins_range=hist_range)
combined_features.append(hist_features)
if hog_feat:
# Call get_hog_features() with vis=False, feature_vec=True
if hog_channel == 'ALL':
hog_features = []
for channel in range(feature_image.shape[2]):
hog_features.append(get_hog_features(feature_image[:,:,channel],
orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True))
hog_features = np.ravel(hog_features)
else:
hog_features = get_hog_features(feature_image[:,:,hog_channel],
orient, pix_per_cell, cell_per_block, vis=False, feature_vec=True)
combined_features.append(hog_features)
# Append the new feature vector to the features list
features.append(np.concatenate(combined_features))
# Return list of feature vectors
return features
# car_features = extract_features(cars)
# notcar_features = extract_features(notcars)
# if len(car_features) > 0:
# # Create an array stack of feature vectors
# X = np.vstack((car_features, notcar_features)).astype(np.float64)
# # Fit a per-column scaler
# X_scaler = StandardScaler().fit(X)
# # Apply the scaler to X
# scaled_X = X_scaler.transform(X)
# # Plot an example of raw and scaled features
# fig = plt.figure(figsize=(12,4))
# plt.subplot(131)
# plt.imshow(car_image)
# plt.title('Original Image')
# plt.subplot(132)
# plt.plot(X[car_ind])
# plt.title('Raw Features')
# plt.subplot(133)
# plt.plot(scaled_X[car_ind])
# plt.title('Normalized Features')
# fig.tight_layout()
# fig.savefig('output_images/extract_feature.png')
# else:
# print('Your function only returns empty feature vectors...')
import time
from sklearn.model_selection import train_test_split
from sklearn.svm import LinearSVC
color_space = 'YCrCb' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9 # HOG orientations
pix_per_cell = 8 # HOG pixels per cell
cell_per_block = 2 # HOG cells per block
hog_channel = "ALL" # Can be 0, 1, 2, or "ALL"
spatial_size = (32, 32) # Spatial binning dimensions
hist_bins = 32 # Number of histogram bins
spatial_feat = True # Spatial features on or off
hist_feat = True # Histogram features on or off
car_features = extract_features(cars, cspace=color_space, spatial_size=spatial_size,
hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel,
spatial_feat=spatial_feat, hist_feat=hist_feat)
notcar_features = extract_features(notcars, cspace=color_space, spatial_size=spatial_size,
hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel,
spatial_feat=spatial_feat, hist_feat=hist_feat)
# Create an array stack of feature vectors
X = np.vstack((car_features, notcar_features)).astype(np.float64)
# Fit a per-column scaler
X_scaler = StandardScaler().fit(X)
# Apply the scaler to X
scaled_X = X_scaler.transform(X)
# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
scaled_X, y, test_size=0.2, random_state=rand_state)
print('Using:',orient,'orientations',pix_per_cell,
'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
# Use a linear SVC
svc = LinearSVC()
# Check the training time for the SVC
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
# Check the prediction time for a single sample
t=time.time()
n_predict = 10
print('My SVC predicts: ', svc.predict(X_test[0:n_predict]))
print('For these',n_predict, 'labels: ', y_test[0:n_predict])
t2 = time.time()
print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
# car_features = extract_features(cars, cspace=color_space, spatial_size=spatial_size,
# hist_bins=hist_bins, hist_range=hist_range,
# orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel,
# spatial_feat=spatial_feat, hist_feat=hist_feat)
# notcar_features = extract_features(notcars, cspace=color_space, spatial_size=spatial_size,
# hist_bins=hist_bins, hist_range=hist_range,
# orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel,
# spatial_feat=spatial_feat, hist_feat=hist_feat)
# X = np.vstack((car_features, notcar_features)).astype(np.float64)
# X_scaler = StandardScaler().fit(X)
# scaled_X = X_scaler.transform(X)
# y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
# svc = LinearSVC()
# svc.fit(scaled_X, y)
import pickle
hist_range = (0, 256)
dist_pickle = {}
dist_pickle["svc"] = svc
dist_pickle["scaler"] = X_scaler
dist_pickle["color_space"] = color_space
dist_pickle["orient"] = orient
dist_pickle["pix_per_cell"] = pix_per_cell
dist_pickle["cell_per_block"] = cell_per_block
dist_pickle["hog_channel"] = hog_channel
dist_pickle["spatial_size"] = spatial_size
dist_pickle["hist_bins"] = hist_bins
dist_pickle["hist_range"] = hist_range
with open('svc_pickle.p', 'wb') as f:
pickle.dump(dist_pickle, f)
import pickle
dist_pickle = pickle.load(open("svc_pickle.p", "rb"))
svc = dist_pickle["svc"]
X_scaler = dist_pickle["scaler"]
color_space = dist_pickle["color_space"]
orient = dist_pickle["orient"]
pix_per_cell = dist_pickle["pix_per_cell"]
cell_per_block = dist_pickle["cell_per_block"]
hog_channel = dist_pickle["hog_channel"]
spatial_size = dist_pickle["spatial_size"]
hist_bins = dist_pickle["hist_bins"]
hist_range = dist_pickle["hist_range"]
# Define a single function that can extract features using hog sub-sampling and make predictions
def find_cars(img, ystart, ystop, scale, svc, X_scaler, color_space, orient, pix_per_cell, cell_per_block, hog_channel, spatial_size, hist_bins, hist_range):
img_tosearch = img[ystart:ystop,:,:]
ctrans_tosearch = convert_color(img_tosearch, conv='RGB2YCrCb')
if scale != 1:
imshape = ctrans_tosearch.shape
ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
# Define blocks and steps as above
nxblocks = (ctrans_tosearch.shape[1] // pix_per_cell)-1
nyblocks = (ctrans_tosearch.shape[0] // pix_per_cell)-1
nfeat_per_block = orient*cell_per_block**2
# 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
window = 64
nblocks_per_window = (window // pix_per_cell)-1
cells_per_step = 2 # Instead of overlap, define how many cells to step
nxsteps = (nxblocks - nblocks_per_window) // cells_per_step
nysteps = (nyblocks - nblocks_per_window) // cells_per_step
# Compute individual channel HOG features for the entire image
if hog_channel == 'ALL':
hog = []
for channel in range(ctrans_tosearch.shape[2]):
hog.append(get_hog_features(ctrans_tosearch[:,:,channel], orient, pix_per_cell, cell_per_block, feature_vec=False))
else:
hog = get_hog_features(ctrans_tosearch[:,:,hog_channel], orient, pix_per_cell, cell_per_block, feature_vec=False)
img_boxes = []
for xb in range(nxsteps):
for yb in range(nysteps):
ypos = yb*cells_per_step
xpos = xb*cells_per_step
# Extract HOG for this patch
if hog_channel == 'ALL':
hog_features = []
for channel in range(ctrans_tosearch.shape[2]):
hog_features.append(hog[channel][ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel())
hog_features = np.hstack(hog_features)
else:
hog_features = hog[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
xleft = xpos*pix_per_cell
ytop = ypos*pix_per_cell
# Extract the image patch
subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64,64))
# Get color features
spatial_features = bin_spatial(subimg, size=spatial_size)
rhist, ghist, bhist, bin_centers, hist_features = color_hist(subimg, nbins=hist_bins)
# Scale features and make a prediction
test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))
test_prediction = svc.predict(test_features)
if test_prediction == 1:
xbox_left = np.int(xleft*scale)
ytop_draw = np.int(ytop*scale)
win_draw = np.int(window*scale)
img_boxes.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))
return img_boxes
# Test Images
ystart = 400
ystop = 656
scale = 1.5
car_image = mpimg.imread('test_images/test6.jpg')
original_image = mpimg.imread('test_images/test5.jpg')
img_boxes = find_cars(car_image, ystart, ystop, scale, svc, X_scaler, color_space, orient, pix_per_cell, cell_per_block, hog_channel, spatial_size, hist_bins, hist_range)
window_image = draw_boxes(car_image, img_boxes)
plt.imshow(window_image)
mpimg.imsave('output_images/window_search.png', window_image)
scales = [1., 1.2, 1.4, 1.6,2.0]
img_boxes = []
for scale in scales:
img_boxes += find_cars(car_image, ystart, ystop, scale, svc, X_scaler, color_space, orient, pix_per_cell, cell_per_block, hog_channel, spatial_size, hist_bins, hist_range)
multiscale_image = draw_boxes(car_image, img_boxes)
plt.imshow(multiscale_image)
mpimg.imsave('output_images/multiscale.png', multiscale_image)
from scipy.ndimage.measurements import label
def add_heat(img, bbox_list):
heatmap = np.zeros_like(img[:,:,0]).astype(np.float)
# Iterate through list of bboxes
for box in bbox_list:
# Add += 1 for all pixels inside each bbox
heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
# Return updated heatmap
return np.clip(heatmap, 0, 255)
def apply_threshold(heatmap, threshold):
heat = np.copy(heatmap)
# Zero out pixels below the threshold
heat[heat <= threshold] = 0
# Return thresholded map
return heat
def draw_labeled_bboxes(img, labels):
# Make a copy of the image
imcopy = np.copy(img)
# Iterate through all detected cars
for car_number in range(1, labels[1]+1):
# Find pixels with each car_number label value
nonzero = (labels[0] == car_number).nonzero()
# Identify x and y values of those pixels
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Define a bounding box based on min/max x and y
bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
# Draw the box on the image
cv2.rectangle(imcopy, bbox[0], bbox[1], (0,0,255), 6)
# Return the image
return imcopy
# Add heat to each box in box list
heatmap = add_heat(car_image, img_boxes)
#plt.imshow(heatmap, cmap='hot')
mpimg.imsave('output_images/heatmap.png', heatmap, cmap='hot')
# Apply threshold to help remove false positives
updated_heatmap = apply_threshold(heatmap, 5)
#plt.figure()
#plt.imshow(updated_heatmap, cmap='hot')
mpimg.imsave('output_images/updated_heatmap.png', updated_heatmap, cmap='hot')
# Find final boxes from heatmap using label function
labels = label(updated_heatmap)
print(labels[1], 'cars found')
draw_img = draw_labeled_bboxes(np.copy(car_image), labels)
# plt.figure()
# plt.imshow(draw_img)
# mpimg.imsave('output_images/final_box.png', draw_img)
plt.imshow(multiscale_image)
plt.title('False positives')
plt.show()
plt.imshow(heatmap, cmap='hot')
plt.title('Heat map of false positives')
plt.show()
plt.imshow(updated_heatmap, cmap='hot')
plt.title('Thresholded Heat Map')
plt.show()
plt.imshow(draw_img)
plt.title('Car Positions')
plt.show()
def pipeline(img):
scales = [1., 1.25, 1.5, 1.75, 2.]
box_list = []
for scale in scales:
box_list += find_cars(img, ystart, ystop, scale, svc, X_scaler, color_space, orient, pix_per_cell, cell_per_block, hog_channel, spatial_size, hist_bins, hist_range)
heatmap = add_heat(img, box_list)
updated_heatmap = apply_threshold(heatmap, 4)
labels = label(updated_heatmap)
result = draw_labeled_bboxes(img, labels)
return result
test_images = glob.glob('test_images/*.jpg')
for idx, file in enumerate(test_images):
img = mpimg.imread(file)
result = pipeline(img)
plt.imshow(result)
plt.figure()
mpimg.imsave('output_images/pipeline'+str(idx)+'.png', result)
### Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
video_output = 'project_video_output.mp4'
clip1 = VideoFileClip("project_video.mp4")
video_clip = clip1.fl_image(pipeline)
%time video_clip.write_videofile(video_output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(video_output))